package com.creativewidgetworks.goldparser.engine;

import java.io.File;
import java.io.FileInputStream;
import java.io.FileReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.Reader;
import java.io.StringReader;
import java.util.Map;
import java.util.Stack;
import java.util.TreeMap;

import com.creativewidgetworks.goldparser.engine.enums.AdvanceMode;
import com.creativewidgetworks.goldparser.engine.enums.CGTRecord;
import com.creativewidgetworks.goldparser.engine.enums.EndingMode;
import com.creativewidgetworks.goldparser.engine.enums.LRActionType;
import com.creativewidgetworks.goldparser.engine.enums.ParseMessage;
import com.creativewidgetworks.goldparser.engine.enums.ParseResult;
import com.creativewidgetworks.goldparser.engine.enums.SymbolType;
import com.creativewidgetworks.goldparser.util.FormatHelper;

/**
 * Parser 
 *
 * This is the main class in the GOLD Parser Engine and is used to perform
 * all duties required to the parsing of a source text string. This class
 * contains the LALR(1) State Machine code, the DFA State Machine code,
 * character table (used by the DFA algorithm) and all other structures and
 * methods needed to interact with the developer.
 * 
 * Dependencies: 
 * @see Group
 * @see GroupList
 * @see FStateList
 * @see LRState
 * @see Position
 * @see Production
 * @see ProductionList
 * @see Reduction
 * @see Symbol
 * @see SymbolList
 * @see Token
 *
 * Note that several class fields are marked as protected instead of private. This was done
 * to avoid having to add getters() used only for testing.  If and when the tests are
 * refactored to use reflection to probe these fields, they can once again be marked
 * protected.
 *
 * @author Devin Cook (http://www.DevinCook.com/GOLDParser)
 * @author Ralph Iden (http://www.creativewidgetworks.com), port to Java
 * @version 5.0.0
 */
public class Parser {
    
    // Standard attribute names
    public static final String ABOUT             = "About";
    public static final String AUTHOR            = "Author";
    public static final String CASE_SENSITIVE    = "Case Sensitive";
    public static final String CHARACTER_MAPPING = "Character Mapping";
    public static final String CHARACTER_SET     = "Character Set";
    public static final String GENERATED_BY      = "Generated By";
    public static final String GENERATED_DATE    = "Generated Date";
    public static final String NAME              = "Name";
    public static final String START_SYMBOL      = "Start Symbol";
    public static final String VERSION           = "Version";
   
    public static final String PARSER_NAME = "GOLD Parser Engine - Version ";
    public static final String PARSER_VERSION = "5.0.3";

    // Flag to indicate which grammar table file is being processed
    protected boolean version1Format;
    
    // Symbols recognized by the system
    protected SymbolList symbolTable;

    // DFA
    protected FAStateList dfa;
    protected CharacterSetList characterSetTable;
    protected StringBuilder lookaheadBuffer;
    
    // Productions
    protected ProductionList productionTable;
    
    // LALR
    protected LRStateList lrStates;
    private int currentLALR;
    protected Stack<Token> stack;

    // Fields for Reductions and errors
    private SymbolList expectedSymbols; 
    protected boolean haveReduction;
    private boolean trimReductions;      
    
    // Locally used fields
    private boolean tablesLoaded;
    private Stack<Token> inputTokens; // Tokens to be analyzed

    // Input reader for the source code to parse
    protected Reader source;
    
    // Line and column information
    private Position sysPosition;      // Internal only, so user cannot alter values
    private Position currentPosition;  // Location of last read terminal

    // Grammar attributes
    protected Map<String, String> attributes;
    
    // Lexical groups
    private Stack<Token> groupStack;
    protected GroupList groupTable;
    
    public Parser() {
        stack = new Stack<Token>();
        inputTokens = new Stack<Token>();
        groupStack = new Stack<Token>();
    }
    
    /*----------------------------------------------------------------------------*/

    /**
     * Return library name and version information
     * @return version information
     */
    public String about() {
        return PARSER_NAME + PARSER_VERSION;
    }

    /*----------------------------------------------------------------------------*/

    /**
     * Consume/remove characters from the front of the lookahead buffer
     * and adjust the value of the system Position object.
     * @param count the number of characters to consume
     */
    private void consumeBuffer(int count) {
        if (count > 0 && count <= lookaheadBuffer.length()) {
            // Adjust position
            for (int i = 0; i < count; i++) {
                char c = lookaheadBuffer.charAt(i);
                if (c == 0x0A) {
                    if (sysPosition.getColumn() > 1) {
                        // Increment row if Unix EOLN (LF)
                        sysPosition.incrementLine();
                    }
                } else if (c == 0x0D) {
                    sysPosition.incrementLine();
                } else {
                    sysPosition.incrementColumn();
                }
            }
            
            // Remove the characters
            lookaheadBuffer.delete(0, count);
        }
    }

    /*----------------------------------------------------------------------------*/

    public String getAttribute(String name) {
        return getAttribute(name, null);
    }
    
    public String getAttribute(String name, String defaultValue) {
        if (attributes == null) {
            attributes = new TreeMap<String, String>();
        }
        
        String value = attributes.get(name);
        return value == null ? defaultValue : value;
    }
    
    public void setAttribute(String name, String value) {
        if (attributes == null) {
            attributes = new TreeMap<String, String>();
        }
        attributes.put(name, value);
    }
    
    /*----------------------------------------------------------------------------*/

    public Position getCurrentPosition() {
        return currentPosition;
    }
    
    /*----------------------------------------------------------------------------*/
    
    /**
     * Return the last token read by the parser
     * @return the last token read by the parser.
     */
    protected Token getCurrentToken() {
        return inputTokens.peek();
    }

    /*----------------------------------------------------------------------------*/

    /**
     * When parse() returns a REDUCE, the method will return the current reduction.
     * @return the reduction
     */
    public Reduction getCurrentReduction() {
        return haveReduction ? stack.peek().asReduction() : null;
    }
    
    protected void setCurrentReduction(Reduction reduction) {
        if (haveReduction) {
            stack.peek().setData(reduction);
        }
    }
    
    /*----------------------------------------------------------------------------*/
    
    /**
     * Return the list of expected symbols
     * @Token the last token read by the parser.
     */
    public SymbolList getExpectedSymbols() {
        return expectedSymbols;
    }
    
    /*----------------------------------------------------------------------------*/
    
    /**
     * Searches the symbol table for the first occurrence of the the
     * specified symbol type.
     * @param SymbolTypeTest to find
     * @return Symbol, the first symbol of type or null if no symbol found
     */
    private Symbol getFirstSymbolOfType(SymbolType type) {
        for (Symbol symbol : symbolTable) {
            if (symbol.getType().equals(type)) {
                return symbol;
            }
        }
        return null;
    }

    /*----------------------------------------------------------------------------*/
    
    /**
     * Searches the symbol table for a specific symbol specified by the symbol name.
     * @param name of symbol to find
     * @return Symbol, the symbol or null if no symbol found
     */
    protected Symbol getSymbolByName(String name) {
        if (symbolTable != null) {
            for (Symbol symbol : symbolTable) {
                if (symbol.getName().equals(name)) {
                    return symbol;
                }
            }
        }
        return null;
    }
    
    /*----------------------------------------------------------------------------*/

    /**
     * Return the parser's current position (row, column)
     * @return Position
     */
    public Position getPosition() {
        return currentPosition;
    }
    
    /*----------------------------------------------------------------------------*/

    protected Reader getSource() {
        return source;
    }
    
    /*----------------------------------------------------------------------------*/

    protected boolean isVersion1Format() {
        return version1Format;
    }
    
    protected void setVersion1Format(boolean version1) {
        version1Format = version1;
    }
    
    /*----------------------------------------------------------------------------*/

    /**
     * Loads the parse tables from the specified file.
     * NOTE: Only CGT version 5.0 is supported.
     * @param file to open and load.
     * @return true if the file was successfully processed.
     * @throws IOException
     */
    protected boolean loadTables(File file) throws IOException {
        if (file == null) {
            throw new IOException(FormatHelper.formatMessage("messages", "error.cgt_missing"));
        }
        return loadTables(new FileInputStream(file));
    }

    /**
     * Loads the parse tables from the specified input stream. The inputstream will
     * be closed when the method returns.
     * NOTE: Only CGT version 5.0 is supported.
     * @param input stream to read.
     * @return true if the stream was successfully processed.
     * @throws IOException
     */
    protected boolean loadTables(InputStream input) throws IOException {
        boolean result = true;
        int index;
            
        CGT cgt = new CGT();
        CharacterSet characterSet;
        
        try {
            cgt.open(input);
            
            restart();
            tablesLoaded = false;
            
            while (result) {
                cgt.getNextRecord();
                if (cgt.atEOF()) {
                    break;
                }

                int recordType = cgt.retrieveByte();
                // System.out.println(CGTRecord.getCGTRecord(recordType));
                
                switch (CGTRecord.getCGTRecord(recordType)) {
                    case PARAMETER:
                        version1Format = true;
                        setAttribute(NAME, cgt.retrieveString());
                        setAttribute(VERSION, cgt.retrieveString());
                        setAttribute(AUTHOR, cgt.retrieveString());
                        setAttribute(ABOUT, cgt.retrieveString());
                        setAttribute(CASE_SENSITIVE, Boolean.toString(cgt.retrieveBoolean()));
                        setAttribute(START_SYMBOL, Integer.toString(cgt.retrieveInteger()));                        
                        break;
                
                    case PROPERTY:
                        // Index (not used), name, value
                        version1Format = false;
                        cgt.retrieveInteger();  // Index (not used)
                        setAttribute(cgt.retrieveString(), cgt.retrieveString());
                        break;
                      
                    // Counts for Symbols, Rules, DFA, and LALR lists    
                    case COUNTS:
                    case COUNTS5:
                        symbolTable = new SymbolList(cgt.retrieveInteger());
                        characterSetTable = new CharacterSetList(cgt.retrieveInteger());
                        productionTable = new ProductionList(cgt.retrieveInteger());
                        dfa = new FAStateList(cgt.retrieveInteger());
                        lrStates = new LRStateList(cgt.retrieveInteger());
                        if (!isVersion1Format()) {
                            groupTable = new GroupList(cgt.retrieveInteger());
                        } else {
                            // Create an empty table in case GROUP_START/END pairs follow
                            groupTable = new GroupList();
                        }
                        break;

                    // Character set     
                    case CHARSET:
                        index = cgt.retrieveInteger();
                        characterSet = new CharacterSet();
                        characterSetTable.set(index, characterSet);     
                        characterSet.add(new CharacterRange(cgt.retrieveString()));
                        break;
                        
                    // Character range     
                    case CHARRANGES:
                        index = cgt.retrieveInteger();
                        cgt.retrieveInteger(); // codepage
                        cgt.retrieveInteger(); // total sets
                        cgt.retrieveEntry(); // reserved

                        characterSet = new CharacterSet();
                        characterSetTable.set(index, characterSet);
                        while (!cgt.isRecordComplete()) {
                            characterSet.add(new CharacterRange(cgt.retrieveInteger(), cgt.retrieveInteger()));
                        }

                        break;                        
                        
                    // Symbols    
                    case SYMBOL:
                        index = cgt.retrieveInteger();
                        String name = cgt.retrieveString();
                        SymbolType type = SymbolType.getSymbolType(cgt.retrieveInteger());
                        Symbol symbol = new Symbol(name, type, index);
                        symbolTable.set(index, symbol);
                        break;

                    // Rules (productions)    
                    case RULE:
                        index = cgt.retrieveInteger();
                        int headIndex = cgt.retrieveInteger();
                        cgt.retrieveEntry();  // Reserved
                        
                        Production production = new Production(symbolTable.get(headIndex), index);
                        productionTable.set(index, production);
                        while (!cgt.isRecordComplete()) {
                            int symIndex = cgt.retrieveInteger();
                            production.getHandle().add(symbolTable.get(symIndex));
                        }
                        
                        break;                        

                    // Initial states for DFA and LALR
                    case INITIALSTATES:
                        dfa.setInitialState(cgt.retrieveInteger());
                        lrStates.setInitialState(cgt.retrieveInteger());
                        break;                        
                        
                    // Groups   
                    case GROUP:
                        index = cgt.retrieveInteger();

                        Group group = new Group();
                        group.setName(cgt.retrieveString());
                        group.setContainer(symbolTable.get(cgt.retrieveInteger()));
                        group.setStart(symbolTable.get(cgt.retrieveInteger()));
                        group.setEnd(symbolTable.get(cgt.retrieveInteger()));
                        group.setAdvanceMode(AdvanceMode.getAdvanceMode(cgt.retrieveInteger()));
                        group.setEndingMode(EndingMode.getEndingMode(cgt.retrieveInteger()));

                        cgt.retrieveEntry(); // Reserved                        
                        
                        // Nesting levels
                        int count = cgt.retrieveInteger();
                        for (int i = 0; i < count; i++) {
                            group.getNesting().add(cgt.retrieveInteger());
                        }
  
                        // Link back
                        group.getContainer().setGroup(group);
                        group.getStart().setGroup(group);
                        group.getEnd().setGroup(group);
                        
                        groupTable.set(index, group);
                        break;
                        
                    case DFASTATE:
                        index = cgt.retrieveInteger();
                        boolean accept = cgt.retrieveBoolean();
                        int acceptIndex = cgt.retrieveInteger();
                        cgt.retrieveEntry(); // Reserved

                        if (accept) {
                            dfa.set(index, new FAState(symbolTable.get(acceptIndex)));
                        } else {
                            dfa.set(index, new FAState());
                        }

                        while (!cgt.isRecordComplete()) {
                            int setIndex = cgt.retrieveInteger();
                            int target = cgt.retrieveInteger();
                            cgt.retrieveEntry();  // Reserved
                            dfa.get(index).getEdges().add(new FAEdge(characterSetTable.get(setIndex), target));
                        }
                        
                        break;
                        
                    case LRSTATE:
                        index = cgt.retrieveInteger();
                        cgt.retrieveEntry();  // Reserved
                        
                        LRState lrState = new LRState();
                        lrStates.set(index, lrState);
                        while (!cgt.isRecordComplete()) {
                            symbol = symbolTable.get(cgt.retrieveInteger());
                            LRActionType actionType = LRActionType.getLRActionType(cgt.retrieveInteger());
                            int value = cgt.retrieveInteger();
                            cgt.retrieveEntry();  // Reserved
                            lrState.add(new LRAction(symbol,actionType, value));
                        }
                        
                        break;
                        
                    case UNDEFINED:
                        throw new IOException("Unknown record type of " + recordType + " was read.");
                }
            }
        } finally {
            cgt.close();
        }
        
        tablesLoaded = result;
        
        resolveCommentGroupsForVersion1Grammars();
        
        return result;
    }
    
    /*----------------------------------------------------------------------------*/

    /**
     * Return a single character at charIndex. This method will read and fill the
     * buffer as needed from the source stream. 
     * @param charIndex offset of the lookahead buffer.
     * @return char that was read or "" EOF has been reached.
     */
    private String lookahead(int charIndex) {
        if (charIndex >= 0) {
            if (charIndex > lookaheadBuffer.length()) {
                // Requesting data past the end of stream, so perform a read
                int readCount = charIndex - lookaheadBuffer.length();
                for (int i = 0; i < readCount; i++) {
                    int c;
                    try {
                        c = source.read();
                    } catch (IOException ioe) {
                        c = -1;
                    }
                    if (c != -1) {
                        lookaheadBuffer.append((char)c);
                    } else {
                        break; // EOF reached
                    }
                }
            }
            
            // If the buffer is still smaller than charIndex, we have reached
            // the end of the text. In this case, return a null string - the DFA
            // code will understand.
            return (charIndex <= lookaheadBuffer.length()) ? String.valueOf(lookaheadBuffer.charAt(charIndex - 1)) : "";
        }
        
        return "";
    }
    
    /*----------------------------------------------------------------------------*/

    /**
     * Return count characters from the lookahead buffer. 
     * These characters are used to create the text stored in a token. Because of  
     * the design of the DFA algorithm, count should never exceed the buffer length.
     * @param count number of characters to return
     * @return String 
     */
    private String getLookaheadBuffer(int count) {
        if (count > lookaheadBuffer.length()) {
            count = lookaheadBuffer.length();
        }
        
        return count > 0 ? lookaheadBuffer.substring(0, count) : "";
    }
    
    /*----------------------------------------------------------------------------*/

    /**
     * This method implements the DFA for the parser's lexer. A
     * token is generated which is used by the LALR state machine.
     * @return Token
     */
    private Token lookaheadDFA() {
        Token token = new Token();
     
        int currentDFA = dfa.getInitialState();
        
        int curPosition = 1;            // Next byte in the input stream
        int lastAcceptState = -1;       // Nothing has been accepted yet
        int lastAcceptPosition = -1;
        int target = 0;
        
        String str = lookahead(1);
        if (str.length() > 0) {
            boolean found;
            boolean done = false;
            while (!done) {
                // Search all the branches of the current DFA state for the next 
                // character in the input stream. If found, the target state is returned.
                str = lookahead(curPosition);
                if (str.length() == 0) {
                    found = false;
                } else {
                    found = false;
                    for (int i = 0; !found && i < dfa.get(currentDFA).getEdges().size(); i++) {
                        FAEdge edge = dfa.get(currentDFA).getEdges().get(i);
                        if (edge.getChars().contains(str.charAt(0))) {
                            found = true;
                            target = edge.getTarget();
                        }
                    }
                }

                // This block checks whether an edge was found from the current state. If so, the 
                // state and current position advance. Otherwise it is time to exit the main loop 
                // and report the token found (if there was one). If the LastAcceptState is -1, then 
                // we never found a match and the Error Token is created. Otherwise, a new token is 
                // created using the Symbol in the Accept State and all the characters that comprise it.
                if (found) {
                    // This code checks whether the target state accepts a token. If so, it sets
                    // the appropriate variables so when the algorithm is done, it can return the
                    // proper token and number of characters.
                    if (dfa.get(target).getAccept() != null) {
                        lastAcceptState = target;
                        lastAcceptPosition = curPosition;
                    }
                    currentDFA = target;
                    curPosition++;
                } else {
                    // No edge found
                    done = true;
                    if (lastAcceptState == -1) {
                        // Lexer doesn't recognize the symbol
                        token.setSymbol(getFirstSymbolOfType(SymbolType.ERROR));
                        token.setData(getLookaheadBuffer(1));
                    } else {
                        // Create Token and read characters
                        // Data contains the total number of accept characters
                        token.setSymbol(dfa.get(lastAcceptState).getAccept());
                        token.setData(getLookaheadBuffer(lastAcceptPosition));
                    }
                }
            }
        } else {
            token.setData("");
            token.setSymbol(getFirstSymbolOfType(SymbolType.END));
        }

        token.setPosition(new Position(sysPosition));
        
        return token;
    }

    /*----------------------------------------------------------------------------*/

    /**
     * Open a file to be parsed
     * @param sourceFile to be parsed
     * @return true if the file is ready to be parsed
     * @throws IOException if file is not available.
     */
    protected boolean open(File sourceFile) throws IOException {
        return open(new FileReader(sourceFile));
    }
    
    /**
     * Prepare the parser to process the source contained in the String.
     * @param sourceStatements, the code to parse
     * @return true if the source is ready to be parsed.
     */
    protected boolean open(String sourceStatements) {
        return open(new StringReader(sourceStatements));
    }
    
    /**
     * Open the reader to be parsed
     * @param reader that will be used to read the source code to parse.
     * @return true if the reader is ready to be parsed.
     */
    protected boolean open(Reader reader) {
        restart();
        source = reader;
        stack.push(new Token());
        return true;
    }

    /*----------------------------------------------------------------------------*/

    /**
     * Returns the next token in the stream -- This method can be overridden to support
     * virtual terminals (indentation sensitive grammars, etc.)
     */
    protected Token nextToken() {
        return produceToken();
    }
    
    /**
     * Performs a parse action on the input stream. This method is typically used in a loop until 
     * either the grammar is accepted or an error occurs.
     * @return ParseMessage
     */
    protected ParseMessage parse() {
        if (!tablesLoaded) {
            return ParseMessage.NOT_LOADED_ERROR;
        }

        Token read;
        ParseMessage parseMessage = ParseMessage.UNDEFINED;
        
        // Loop until a breakable event
        boolean done = false;
        while (!done) {
            if (inputTokens.size() == 0) {
                read = nextToken();
                inputTokens.push(read);
                
                // Handle the case where an unterminated comment block consumes the entire program
                if (SymbolType.END.equals(read.getType()) && groupStack.size() > 0) {
                    // Runaway group
                    parseMessage = ParseMessage.GROUP_ERROR;                    
                } else {
                    // A good token was read
                    parseMessage = ParseMessage.TOKEN_READ;
                }
                
                done = true;
            } else {
                read = inputTokens.peek();
                currentPosition.set(read.getPosition());  // Update current position

                if (SymbolType.NOISE.equals(read.getType())) {
                    // Discard token - these tokens were already reported to the user
                    inputTokens.pop();
                } else if (SymbolType.ERROR.equals(read.getType())) {
                    parseMessage = ParseMessage.LEXICAL_ERROR;
                    done = true;
                } else if (SymbolType.END.equals(read.getType()) && groupStack.size() > 0) {
                    // Runaway group
                    parseMessage = ParseMessage.GROUP_ERROR;
                    done = true;                    
                } else {
                    ParseResult parseResult = parseLALR(read);  // Same method as v1
                    switch (parseResult) {
                        case ACCEPT:
                            parseMessage = ParseMessage.ACCEPT;
                            done = true;
                            break;
                            
                        case INTERNAL_ERROR:
                            parseMessage = ParseMessage.INTERNAL_ERROR;
                            done = true;
                            break;

                        case REDUCE_NORMAL:
                            parseMessage = ParseMessage.REDUCTION;
                            done = true;
                            break;

                        case SHIFT:
                            // ParseToken() shifted the token on the front of the Token-Queue. It 
                            // now exists on the Token-Stack and must be eliminated from the queue.
                            inputTokens.remove(0);
                            break;
                            
                        case SYNTAX_ERROR:
                            parseMessage = ParseMessage.SYNTAX_ERROR;
                            done = true;
                            break;
                            
                        case REDUCE_ELIMINATED:  // fall through intended
                        case UNDEFINED:
                            // do nothing
                            break;
                    }
                }
            }
        }
        
        return parseMessage;
    }
        
    /*----------------------------------------------------------------------------*/

    /**
     * This method analyzes a token and either:
     *   1. Makes a SINGLE reduction and pushes a complete Reduction object on the stack
     *   2. Accepts the token and shifts
     *   3. Errors and places the expected symbol indexes in the Tokens list.
     *   
     * @param nextToken to be analyzed
     * @return ParseResult  
     */
    private ParseResult parseLALR(Token nextToken) {
        ParseResult parseResult = null;

        haveReduction = false;
        
        LRAction parseAction = lrStates.get(currentLALR).find(nextToken);
        //System.out.println("Action: " + parseAction.toString());
        
        switch (parseAction.getType()) {
            case ACCEPT:
                haveReduction = true;
                parseResult = ParseResult.ACCEPT;
                break;
                
            case REDUCE:
                // Produce a reduction - remove as many tokens as members in the rule and push a nonterminal token
                Production production = productionTable.get(parseAction.getValue());
                
                Token head;
                if (trimReductions && production.containsOneNonTerminal()) {
                    // The current rule only consists of a single nonterminal and can be trimmed from the
                    // parse tree. Usually we create a new Reduction, assign it to the Data property
                    // of Head and push it on the m_Stack. However, in this case, the Data property of the
                    // Head will be assigned the Data property of the reduced token (i.e. the only one
                    // on the m_Stack). To save code, the value popped of the m_Stack is changed into the head.
                    head = stack.pop();
                    head.setSymbol(production.getHead());
                    parseResult = ParseResult.REDUCE_ELIMINATED;
                } else {
                    haveReduction = true;
                    Reduction newReduction = new Reduction(production.getHandle().size());
                    newReduction.setParent(production);
                    for (int i = production.getHandle().size() - 1; i >= 0; i--) {
                        newReduction.set(i, stack.pop());
                    }
                    head = new Token(production.getHead(), newReduction);
                    parseResult = ParseResult.REDUCE_NORMAL;
                }
                
                // goto value
                int index = stack.peek().getState();

                LRAction lrAction = lrStates.get(index).find(production.getHead());
                if (!lrAction.equals(LRState.LRACTION_UNDEFINED)) {
                    currentLALR = lrAction.getValue();
                    head.setState(currentLALR);
                    stack.push(head);
                } else {
                    parseResult = ParseResult.INTERNAL_ERROR;
                }
                
                break;
                
            case SHIFT:
                currentLALR = parseAction.getValue();
                nextToken.setState(currentLALR);
                stack.push(nextToken);
                parseResult = ParseResult.SHIFT;
                break;
                
            case ERROR:      // fall-through intended
            case GOTO:       // fall-through intended         
            case UNDEFINED:
                // Syntax error - produce a list of expected symbols to report
                expectedSymbols.clear();
                for (LRAction action : lrStates.get(currentLALR)) {
                    SymbolType type = action.getSymbol().getType();
                    switch (type) {
                        case CONTENT:       // fall-through intended
                        case END:           // fall-through intended
                            expectedSymbols.add(action.getSymbol());
                            break;

                        case GROUP_START:   // fall-through intended
                        case GROUP_END:     // fall-through intended
                        case COMMENT_LINE:
                            expectedSymbols.add(action.getSymbol());
                            break;
                            
                        case ERROR:
                        case NOISE:
                        case NON_TERMINAL:
                        case UNDEFINED:
                            // do nothing
                            break;
                    }
                }
                
                parseResult = ParseResult.SYNTAX_ERROR;
                break;
        }
        
        return parseResult;
    }
    
    /*----------------------------------------------------------------------------*/

    /**
     * This method creates a token and also takes into account the current
     * lexing mode of the parser. In particular, it contains the group logic. 
     *
     * A stack is used to track the current "group". This replaces the comment
     * level counter. Text is appended to the token on the top of the stack. This 
     * allows the group text to returned in one chunk.
     * @return Token
     */
    protected Token produceToken() {
        Token token = null;

        boolean nestGroup = false;
        
        boolean done = false;
        while (!done) {
            Token read = lookaheadDFA();
            
            // Groups (comments, etc.)
            // The logic - to determine if a group should be nested - requires that the top 
            // of the stack and the symbol's linked group need to be looked at. Both of these 
            // can be unset. So, this section sets a boolean and avoids errors. We will use 
            // this boolean in the logic chain below. 
            if (read.getType().equals(SymbolType.GROUP_START) || read.getType().equals(SymbolType.COMMENT_LINE)) {
                if (groupStack.size() == 0) {
                    nestGroup = true;
                } else {
                    nestGroup = groupStack.peek().getGroup().getNesting().contains(read.getGroup().getIndex());
                }
            } else {
                nestGroup = false;
            }

            // Logic chain
            if (nestGroup) {
                consumeBuffer(read.asString().length());
                
                // fix up the comment block
                if (read.getData() != null) {
                    read.appendData(read.getData().toString());
                    read.setData(null);
                }
                
                groupStack.push(read);                
            } else if (groupStack.size() == 0) {
                // The token is ready to be analyzed
                consumeBuffer(read.asString().length());
                token = read;
                done = true;
            } else if (groupStack.peek().getGroup().getEnd().getTableIndex() == read.getTableIndex()) {
                // End the current group
                Token pop = groupStack.pop();

                // Ending logic
                if (pop.getGroup().getEndingMode() == EndingMode.CLOSED) {
                    pop.appendData(read.asString());
                    consumeBuffer(read.asString().length());
                }
                
                if (groupStack.size() == 0) {
                    // We are out of the group. Return pop'd token which contains all the group text    
                    pop.setSymbol(pop.getGroup().getContainer());
                    token = pop;
                    done = true;
                } else {
                    // Append group text to parent
                    groupStack.peek().appendData(pop.asString());
                }
            } else if (read.getType().equals(SymbolType.END)) {
                // EOF always stops the loop. The caller method (parse) can flag a runaway group error.
                token = read;
                done = true;
            } else {
                // We are in a group, Append to the Token on the top of the stack.
                // Take into account the Token group mode 
                Token top = groupStack.peek();
                if (top.getGroup().getAdvanceMode() == AdvanceMode.TOKEN) {
                    // Append all text
                    top.appendData(read.asString());
                    consumeBuffer(read.asString().length());
                } else {
                    // Append one character
                    top.appendData(read.asString().substring(0, 1));
                    consumeBuffer(1);
                }
            }                
        }
        
        return token;
    }
    
    /*----------------------------------------------------------------------------*/

    /**
     * Inserts Group objects into the group table so comments can be processed in a 
     * grammar.  It is assumed that version 1.0 files have a maximum of 1 closed
     * comment block and one comment line symbol.
     */
    private void resolveCommentGroupsForVersion1Grammars() {
        if (isVersion1Format()) {
            Group group;
            Symbol symbolStart = null;
            Symbol symbolEnd = null;
            
            // Create a new COMMENT_LINE group
            for (Symbol currentStartSymbol : symbolTable) {
                if (currentStartSymbol.getType().equals(SymbolType.COMMENT_LINE)) {
                    symbolStart = currentStartSymbol;
                    group = new Group();
                    group.setName("Comment Line");
                    group.setContainer(symbolTable.findByName(SymbolList.SYMBOL_COMMENT));
                    group.setStart(symbolStart);
                    group.setEnd(symbolTable.findByName("NewLine"));
                    group.setAdvanceMode(AdvanceMode.TOKEN);
                    group.setEndingMode(EndingMode.OPEN);
                    groupTable.add(group);
                    symbolStart.setGroup(group);
                    break;
                }
            }

            // Create a new COMMENT_BLOCK group
            for (Symbol currentStartSymbol : symbolTable) {
                if (currentStartSymbol.getType().equals(SymbolType.GROUP_START)) {
                    symbolStart = symbolEnd = currentStartSymbol;
                    for (Symbol currentEndSymbol : symbolTable) {
                        if (currentEndSymbol.getType().equals(SymbolType.GROUP_END)) { 
                            symbolEnd = currentEndSymbol;
                            break;
                        }
                    }    
                    group = new Group();
                    group.setName("Comment Block");
                    group.setContainer(symbolTable.findByName(SymbolList.SYMBOL_COMMENT));
                    group.setStart(symbolStart);
                    group.setEnd(symbolEnd);
                    group.setAdvanceMode(AdvanceMode.TOKEN);
                    group.setEndingMode(EndingMode.CLOSED);
                    groupTable.add(group);
                    
                    symbolStart.setGroup(group);                         
                    symbolEnd.setGroup(group);                         
                    
                    break;
                }
            }
        }
    }
    
    /*----------------------------------------------------------------------------*/
    
    /**
     * Restarts the parser. The loaded tables are retained
     */
    protected void restart() {
        currentLALR = LRState.INITIAL_STATE;
        
        sysPosition = new Position(1, 1);
        currentPosition = new Position(1, 1);
        
        lookaheadBuffer = new StringBuilder();
        
        haveReduction = false;
        
        if (expectedSymbols == null) {
            expectedSymbols = new SymbolList();
        }
        expectedSymbols.clear();

        if (groupStack == null) {
            groupStack = new Stack<Token>();
        }
        groupStack.clear();

        if (inputTokens == null) {
            inputTokens = new Stack<Token>();
        }
        inputTokens.clear();

        if (stack == null) {
            stack = new Stack<Token>();
        }
        stack.clear();
    }
    
    /*----------------------------------------------------------------------------*/

    public void setTrimReductions(boolean value) {
        trimReductions = value;
    }
}
